# coding=utf8


import pandas as pd
from urllib.parse import quote_plus
from pymongo import MongoClient
from random import sample
from threading import Thread

from qichacha_log import get_logger
import logging
# from recrusive_dict_leaves import myprint
from coordinatransfer import bd09_to_gcj02
import aiohttp
from requests import session
import random
from thread_dynam_production import extract_text_td
from lxml.html import tostring

FORMATTER = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s")
scrawl_bug_logger = get_logger("qichacha")
uri = "mongodb://%s:%s@%s" % (quote_plus('zane'), quote_plus('*#06#'), '121.9.245.183')
client = MongoClient(uri)
db = client['Homestay']

ditt = {}


def myprint(d):
    global ditt
    for k, v in d.items():
        if isinstance(v, dict):
            myprint(v)
        else:
            ditt[k] = v
    return ditt


list_collections = ["house_airbnb",
                    "house_mayi",
                    "house_muniao",
                    "house_pig2",
                    "house_tujia", ]


list_columns = ['hid',
                'id',
                'insert_time',
                'house_name',
                'house_type',
                'house_facility',
                'bathroom_count',
                'room_count',
                'bed_count',
                'live_count',
                'release_time',
                'house_info',
                'start_time',
                'end_time',
                'order_or_not',
                'longitude',
                'latitude',
                'address',
                'address_filter',
                'city',
                'province',
                'district',
                'zoning_code',
                'landlord_name',
                'registered_time',
                'original_price',
                'discount_price',
                'other_price',
                'score',
                'highest_score', ]


def parse_dict(dicc):
    redic = {}
    for k, v in dicc.items():
        if not isinstance(v, dict):
            redic[k] = v
        else:
            for kk, vv in v.items():
                if not isinstance(vv, dict):
                    redic[kk] = vv
                else:
                    for kkk, vvv in vv.items():
                        if not isinstance(vvv, dict):
                            redic[kkk] = vvv
    return redic

# list_collections = [
#     "house_mayi",
#     "house_muniao",
#     "house_pig2",
# ]
for collection in list_collections:
    cursor = db[collection].find({})
    df = pd.DataFrame(list(cursor))
    df_detail = df['details_data'].apply(lambda k: parse_dict(k))
    df_detail = df_detail.apply(pd.Series)
    df_detail['id'] = df['id']
    df_detail.to_pickle("/home/mm/Documents/bnb_data_wrangler/{}.pkl".format(collection.split("_")[1] + "_details"))
    print("this is done {}".format(collection))
    # df_detail = pd.read_pickle("/home/mm/Documents/bnb_data_wrangler/test_detailss.pkl")

# for collection in list_collections:
#     sql_mongo = ""
#     cursor = db[collection].find({})
#     df = pd.DataFrame(list(cursor))
